import numpy as np
import os
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, GRU
from keras.preprocessing.image import ImageDataGenerator
from PIL import Image
import scipy.io
import os
import matplotlib.pyplot as plt
import csv
import keras
from keras.preprocessing.image import ImageDataGenerator
from keras.applications import densenet
from keras.models import Sequential, Model, load_model
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, Callback
from keras import regularizers
from keras import backend as k
import tensorflow_hub as hub
import tensorflow as tf
from tensorflow.keras.layers import GlobalAveragePooling2D
from keras.optimizers import SGD, Adam
from tensorflow.keras.models import load_model
#from classification_models.keras import Classifiers
Paper Critique
f = open('/Users/lmschwenke/Downloads/bugs/classes.txt')
label = []
name = []
for line in f.readlines():
label.append(int(line.split()[0]))
name.append(' '.join(line.split()[1:]))
classes = pd.DataFrame([label, name]).T
classes.columns = ['label','name']
classes
| label | name | |
|---|---|---|
| 0 | 1 | rice leaf roller |
| 1 | 2 | rice leaf caterpillar |
| 2 | 3 | paddy stem maggot |
| 3 | 4 | asiatic rice borer |
| 4 | 5 | yellow rice borer |
| ... | ... | ... |
| 97 | 98 | Chlumetia transversa |
| 98 | 99 | Mango flat beak leafhopper |
| 99 | 100 | Rhytidodera bowrinii white |
| 100 | 101 | Sternochetus frigidus |
| 101 | 102 | Cicadellidae |
102 rows × 2 columns
train_df = pd.read_csv('/Users/lmschwenke/Downloads/bugs/train.txt',sep=' ',header=None, engine='python')
train_df.columns = ['image_path','labels']
test_df = pd.read_csv('/Users/lmschwenke/Downloads/bugs/test.txt',sep=' ',header=None, engine='python')
test_df.columns = ['image_path','labels']
Balance the dataset by keeping 40 images per class in the Train dataset and 10 images per class in the Test dataset. Subsetting the data in this way also makes the data into an 80% Train / 20% Test split ratio.
def keep_top_n_records_per_label(df, n):
result_df = pd.DataFrame()
for label in df['labels'].unique():
label_df = df[df['labels'] == label].head(n)
result_df = pd.concat([result_df, label_df])
return result_df
# Balance the dataset AND make into 80% Train and 20% Test
train_df = keep_top_n_records_per_label(train_df, n=40)
test_df = keep_top_n_records_per_label(test_df, n=10)
train_df.groupby('labels').size().reset_index(name='count').head(5)
| labels | count | |
|---|---|---|
| 0 | 0 | 40 |
| 1 | 1 | 40 |
| 2 | 2 | 40 |
| 3 | 3 | 40 |
| 4 | 4 | 40 |
test_df.groupby('labels').size().reset_index(name='count').head(5)
| labels | count | |
|---|---|---|
| 0 | 0 | 10 |
| 1 | 1 | 10 |
| 2 | 2 | 10 |
| 3 | 3 | 10 |
| 4 | 4 | 10 |
train_data_dir = '/Users/lmschwenke/Downloads/bugs/classification/train'
test_data_dir = '/Users/lmschwenke/Downloads/bugs/classification/test'
def delete_extra_files(main_path, file_list):
for root, directories, files in os.walk(main_path):
for file in files:
if file not in file_list:
os.remove(os.path.join(root, file))
delete_extra_files(main_path='/Users/lmschwenke/Downloads/bugs/classification/train',
file_list=list(train_df.image_path))
base_directory = train_data_dir
# Get a list of folder names in the base directory
folders = os.listdir(base_directory)
# Loop through each folder and count the number of files
for folder in folders:
folder_path = os.path.join(base_directory, folder)
if os.path.isdir(folder_path):
num_files = len(os.listdir(folder_path))
print(f"Folder '{folder}' contains {num_files} files.")
Folder '61' contains 40 files. Folder '95' contains 40 files. Folder '59' contains 40 files. Folder '92' contains 40 files. Folder '66' contains 40 files. Folder '50' contains 40 files. Folder '68' contains 40 files. Folder '57' contains 40 files. Folder '32' contains 40 files. Folder '35' contains 40 files. Folder '69' contains 40 files. Folder '56' contains 40 files. Folder '51' contains 40 files. Folder '58' contains 40 files. Folder '67' contains 40 files. Folder '93' contains 40 files. Folder '94' contains 40 files. Folder '60' contains 40 files. Folder '34' contains 40 files. Folder '33' contains 40 files. Folder '20' contains 40 files. Folder '18' contains 40 files. Folder '27' contains 40 files. Folder '9' contains 40 files. Folder '0' contains 40 files. Folder '11' contains 40 files. Folder '7' contains 40 files. Folder '29' contains 40 files. Folder '16' contains 40 files. Folder '42' contains 40 files. Folder '89' contains 40 files. Folder '45' contains 40 files. Folder '73' contains 40 files. Folder '87' contains 40 files. Folder '80' contains 40 files. Folder '74' contains 40 files. Folder '6' contains 40 files. Folder '28' contains 40 files. Folder '17' contains 40 files. Folder '1' contains 40 files. Folder '10' contains 40 files. Folder '19' contains 40 files. Folder '26' contains 40 files. Folder '8' contains 40 files. Folder '21' contains 40 files. Folder '75' contains 40 files. Folder '81' contains 40 files. Folder '86' contains 40 files. Folder '72' contains 40 files. Folder '44' contains 40 files. Folder '43' contains 40 files. Folder '88' contains 40 files. Folder '38' contains 40 files. Folder '36' contains 40 files. Folder '31' contains 40 files. Folder '91' contains 40 files. Folder '65' contains 40 files. Folder '62' contains 40 files. Folder '96' contains 40 files. Folder '100' contains 40 files. Folder '54' contains 40 files. Folder '98' contains 40 files. Folder '53' contains 40 files. Folder '30' contains 40 files. Folder '37' contains 40 files. Folder '39' contains 40 files. Folder '99' contains 40 files. Folder '52' contains 40 files. Folder '101' contains 40 files. Folder '55' contains 40 files. Folder '97' contains 40 files. Folder '63' contains 40 files. Folder '64' contains 40 files. Folder '90' contains 40 files. Folder '46' contains 40 files. Folder '79' contains 40 files. Folder '41' contains 40 files. Folder '83' contains 40 files. Folder '77' contains 40 files. Folder '48' contains 40 files. Folder '70' contains 40 files. Folder '84' contains 40 files. Folder '24' contains 40 files. Folder '23' contains 40 files. Folder '4' contains 40 files. Folder '15' contains 40 files. Folder '3' contains 40 files. Folder '12' contains 40 files. Folder '85' contains 40 files. Folder '71' contains 40 files. Folder '76' contains 40 files. Folder '82' contains 40 files. Folder '49' contains 40 files. Folder '40' contains 40 files. Folder '47' contains 40 files. Folder '78' contains 40 files. Folder '2' contains 40 files. Folder '13' contains 40 files. Folder '5' contains 40 files. Folder '14' contains 40 files. Folder '22' contains 40 files. Folder '25' contains 40 files.
delete_extra_files(main_path='/Users/lmschwenke/Downloads/bugs/classification/test',
file_list=list(test_df.image_path))
base_directory = test_data_dir
# Get a list of folder names in the base directory
folders = os.listdir(base_directory)
# Loop through each folder and count the number of files
for folder in folders:
folder_path = os.path.join(base_directory, folder)
if os.path.isdir(folder_path):
num_files = len(os.listdir(folder_path))
print(f"Folder '{folder}' contains {num_files} files.")
Folder '61' contains 10 files. Folder '95' contains 10 files. Folder '59' contains 10 files. Folder '92' contains 10 files. Folder '66' contains 10 files. Folder '50' contains 10 files. Folder '68' contains 10 files. Folder '57' contains 10 files. Folder '32' contains 10 files. Folder '35' contains 10 files. Folder '69' contains 10 files. Folder '56' contains 10 files. Folder '51' contains 10 files. Folder '58' contains 10 files. Folder '67' contains 10 files. Folder '93' contains 10 files. Folder '94' contains 10 files. Folder '60' contains 10 files. Folder '34' contains 10 files. Folder '33' contains 10 files. Folder '20' contains 10 files. Folder '18' contains 10 files. Folder '27' contains 10 files. Folder '9' contains 10 files. Folder '0' contains 10 files. Folder '11' contains 10 files. Folder '7' contains 10 files. Folder '29' contains 10 files. Folder '16' contains 10 files. Folder '42' contains 10 files. Folder '89' contains 10 files. Folder '45' contains 10 files. Folder '73' contains 10 files. Folder '87' contains 10 files. Folder '80' contains 10 files. Folder '74' contains 10 files. Folder '6' contains 10 files. Folder '28' contains 10 files. Folder '17' contains 10 files. Folder '1' contains 10 files. Folder '10' contains 10 files. Folder '19' contains 10 files. Folder '26' contains 10 files. Folder '8' contains 10 files. Folder '21' contains 10 files. Folder '75' contains 10 files. Folder '81' contains 10 files. Folder '86' contains 10 files. Folder '72' contains 10 files. Folder '44' contains 10 files. Folder '43' contains 10 files. Folder '88' contains 10 files. Folder '38' contains 10 files. Folder '36' contains 10 files. Folder '31' contains 10 files. Folder '91' contains 10 files. Folder '65' contains 10 files. Folder '62' contains 10 files. Folder '96' contains 10 files. Folder '100' contains 10 files. Folder '54' contains 10 files. Folder '98' contains 10 files. Folder '53' contains 10 files. Folder '30' contains 10 files. Folder '37' contains 10 files. Folder '39' contains 10 files. Folder '99' contains 10 files. Folder '52' contains 10 files. Folder '101' contains 10 files. Folder '55' contains 10 files. Folder '97' contains 10 files. Folder '63' contains 10 files. Folder '64' contains 10 files. Folder '90' contains 10 files. Folder '46' contains 10 files. Folder '79' contains 10 files. Folder '41' contains 10 files. Folder '83' contains 10 files. Folder '77' contains 10 files. Folder '48' contains 10 files. Folder '70' contains 10 files. Folder '84' contains 10 files. Folder '24' contains 10 files. Folder '23' contains 10 files. Folder '4' contains 10 files. Folder '15' contains 10 files. Folder '3' contains 10 files. Folder '12' contains 10 files. Folder '85' contains 10 files. Folder '71' contains 10 files. Folder '76' contains 10 files. Folder '82' contains 10 files. Folder '49' contains 10 files. Folder '40' contains 10 files. Folder '47' contains 10 files. Folder '78' contains 10 files. Folder '2' contains 10 files. Folder '13' contains 10 files. Folder '5' contains 10 files. Folder '14' contains 10 files. Folder '22' contains 10 files. Folder '25' contains 10 files.
counts_train = train_df.groupby('labels').size().reset_index(name='count')
plt.bar(counts_train['labels'], counts_train['count'])
plt.xlabel('Label')
plt.ylabel('Count')
plt.title('Counts of Records for Each Label - Train Set')
#plt.xticks(counts_train['label']) # Optional: Set the x-axis tick labels to the unique labels
plt.show()
def count_jpg_files(directory_path):
jpg_count = 0
for root, _, files in os.walk(directory_path):
for file in files:
if file.lower().endswith(".jpg"):
jpg_count += 1
return jpg_count
print('Train Files:',count_jpg_files(train_data_dir))
print('Test Files:',count_jpg_files(test_data_dir))
print('Percent Split of Train:',count_jpg_files(train_data_dir)/(count_jpg_files(train_data_dir)+count_jpg_files(test_data_dir)))
print('Percent Split of Test:',count_jpg_files(test_data_dir)/(count_jpg_files(train_data_dir)+count_jpg_files(test_data_dir)))
Train Files: 4080 Test Files: 1020 Percent Split of Train: 0.8 Percent Split of Test: 0.2
datagen = ImageDataGenerator(rotation_range=10,
rescale = 1.0/255.0,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
vertical_flip=False,
zoom_range=0.1,
shear_range=0.1,
brightness_range=[0.8, 1.2],
fill_mode='nearest')
trainimagedata = datagen.flow_from_directory("/Users/lmschwenke/Downloads/bugs/classification/train",
batch_size = 512,
class_mode = 'categorical',
target_size=(64,64),
subset = 'training'
)
testimagedata = datagen.flow_from_directory("/Users/lmschwenke/Downloads/bugs/classification/test",
batch_size = 256,
class_mode = 'categorical',
target_size=(64,64)
)
valimagedata = datagen.flow_from_directory("/Users/lmschwenke/Downloads/bugs/classification/val",
batch_size = 256,
class_mode = 'categorical',
target_size=(64,64),
subset = 'training'
) # Not subsetted
Found 4080 images belonging to 102 classes. Found 1020 images belonging to 102 classes. Found 7508 images belonging to 102 classes.
# Model Architecture
input_shape = trainimagedata.image_shape
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Conv2D(128,(3,3), input_shape = input_shape,activation='relu',padding='same'))
model.add(tf.keras.layers.MaxPool2D(2,2))
model.add(tf.keras.layers.Conv2D(64,(3,3),activation='relu'))
model.add(tf.keras.layers.MaxPool2D(2,2))
model.add(tf.keras.layers.Flatten())
model.add(Dense(256,activation = 'relu'))
model.add(Dropout(0.25))
model.add(Dense(128,activation = 'relu'))
model.add(Dropout(0.25))
model.add(Dense(102,activation = 'softmax'))
from tensorflow.keras.optimizers import Adam
optimizer = Adam(lr=0.001,beta_1 = 0.9 ,beta_2= 0.999 , epsilon = 1e-07)
WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`. WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.
model.compile(optimizer = optimizer,
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.
model.summary()
Model: "sequential_5"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_2 (Conv2D) (None, 64, 64, 128) 3584
max_pooling2d_2 (MaxPoolin (None, 32, 32, 128) 0
g2D)
conv2d_3 (Conv2D) (None, 30, 30, 64) 73792
max_pooling2d_3 (MaxPoolin (None, 15, 15, 64) 0
g2D)
flatten_1 (Flatten) (None, 14400) 0
dense_9 (Dense) (None, 256) 3686656
dropout_3 (Dropout) (None, 256) 0
dense_10 (Dense) (None, 128) 32896
dropout_4 (Dropout) (None, 128) 0
dense_11 (Dense) (None, 102) 13158
=================================================================
Total params: 3810086 (14.53 MB)
Trainable params: 3810086 (14.53 MB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________
import visualkeras
from PIL import ImageFont
visualkeras.layered_view(model, legend=True)
from keras.callbacks import EarlyStopping
early_stop = EarlyStopping(monitor = 'val_loss', patience =3)
# Fitting the model
with tf.device('/device:GPU:0'):
custom_cnn_history = model.fit(trainimagedata,
validation_data = testimagedata,
epochs= 5,
batch_size=512,
callbacks=[early_stop])
Epoch 1/5 8/8 [==============================] - 14s 2s/step - loss: 4.7799 - accuracy: 0.0093 - val_loss: 4.6312 - val_accuracy: 0.0108 Epoch 2/5 8/8 [==============================] - 13s 2s/step - loss: 4.6344 - accuracy: 0.0115 - val_loss: 4.6239 - val_accuracy: 0.0118 Epoch 3/5 8/8 [==============================] - 13s 2s/step - loss: 4.6242 - accuracy: 0.0088 - val_loss: 4.6212 - val_accuracy: 0.0078 Epoch 4/5 8/8 [==============================] - 12s 2s/step - loss: 4.6217 - accuracy: 0.0098 - val_loss: 4.6170 - val_accuracy: 0.0118 Epoch 5/5 8/8 [==============================] - 12s 2s/step - loss: 4.6248 - accuracy: 0.0098 - val_loss: 4.6152 - val_accuracy: 0.0137
plt.plot(custom_cnn_history.history['loss'])
plt.plot(custom_cnn_history.history['val_loss'])
plt.title('Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.show()
plt.plot(custom_cnn_history.history['accuracy'])
plt.plot(custom_cnn_history.history['val_accuracy'])
plt.title('Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.show()
model.save("/Users/lmschwenke/Downloads/bugs/cnn_model.h5")
/Users/lmschwenke/miniconda3/lib/python3.10/site-packages/keras/src/engine/training.py:3000: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.
saving_api.save_model(
The given model is a sequential neural network with a total of 7 layers. It begins with two convolutional layers followed by max-pooling layers for downsampling. The next steps include a flatten layer and two dense layers, each with dropout regularization to mitigate overfitting. The model concludes with a final dense layer with 102 neurons, representing the number of output classes. The total number of parameters in this model is 3,790,086.
Manual Parameter Calculation:
Dense layer 2: input_shape = 256 (output shape from the previous Dense layer) output_shape = 128 Params = (256 + 1) * 128 = 32896
Dropout layer 2: (Dropout layers have no trainable parameters, so params = 0)
Dense layer 3: input_shape = 128 (output shape from the previous Dense layer) output_shape = 102 Params = (128 + 1) * 102 = 13158
Now, add up the parameters from all the layers:
Total params = 3584 + 73792 + 3686656 + 32896 + 13158 = 3,790,086
from tensorflow.keras.applications.resnet50 import ResNet50
model_name='ResNet50'
base_model=tf.keras.applications.resnet50.ResNet50(include_top=False,
weights="imagenet",
input_shape=trainimagedata.image_shape,
pooling='max')
# Freeze Layers
for layer in base_model.layers[:20]:
layer.trainable = False
# Add Custom Layers
x=base_model.output
x=keras.layers.BatchNormalization(axis=-1,
momentum=0.99,
epsilon=0.001 )(x)
x = Dense(512, kernel_regularizer = regularizers.l2(l = 0.016),
activity_regularizer=regularizers.l1(0.006),
bias_regularizer=regularizers.l1(0.006) ,activation='relu')(x)
x=Dropout(rate=.2, seed=345)(x)
output=Dense(102, activation='softmax')(x)
model=Model(inputs=base_model.input, outputs=output)
model.compile(optimizer = optimizer,
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.
model.summary()
Model: "model_8"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_12 (InputLayer) [(None, 64, 64, 3)] 0 []
conv1_pad (ZeroPadding2D) (None, 70, 70, 3) 0 ['input_12[0][0]']
conv1_conv (Conv2D) (None, 32, 32, 64) 9472 ['conv1_pad[0][0]']
conv1_bn (BatchNormalizati (None, 32, 32, 64) 256 ['conv1_conv[0][0]']
on)
conv1_relu (Activation) (None, 32, 32, 64) 0 ['conv1_bn[0][0]']
pool1_pad (ZeroPadding2D) (None, 34, 34, 64) 0 ['conv1_relu[0][0]']
pool1_pool (MaxPooling2D) (None, 16, 16, 64) 0 ['pool1_pad[0][0]']
conv2_block1_1_conv (Conv2 (None, 16, 16, 64) 4160 ['pool1_pool[0][0]']
D)
conv2_block1_1_bn (BatchNo (None, 16, 16, 64) 256 ['conv2_block1_1_conv[0][0]']
rmalization)
conv2_block1_1_relu (Activ (None, 16, 16, 64) 0 ['conv2_block1_1_bn[0][0]']
ation)
conv2_block1_2_conv (Conv2 (None, 16, 16, 64) 36928 ['conv2_block1_1_relu[0][0]']
D)
conv2_block1_2_bn (BatchNo (None, 16, 16, 64) 256 ['conv2_block1_2_conv[0][0]']
rmalization)
conv2_block1_2_relu (Activ (None, 16, 16, 64) 0 ['conv2_block1_2_bn[0][0]']
ation)
conv2_block1_0_conv (Conv2 (None, 16, 16, 256) 16640 ['pool1_pool[0][0]']
D)
conv2_block1_3_conv (Conv2 (None, 16, 16, 256) 16640 ['conv2_block1_2_relu[0][0]']
D)
conv2_block1_0_bn (BatchNo (None, 16, 16, 256) 1024 ['conv2_block1_0_conv[0][0]']
rmalization)
conv2_block1_3_bn (BatchNo (None, 16, 16, 256) 1024 ['conv2_block1_3_conv[0][0]']
rmalization)
conv2_block1_add (Add) (None, 16, 16, 256) 0 ['conv2_block1_0_bn[0][0]',
'conv2_block1_3_bn[0][0]']
conv2_block1_out (Activati (None, 16, 16, 256) 0 ['conv2_block1_add[0][0]']
on)
conv2_block2_1_conv (Conv2 (None, 16, 16, 64) 16448 ['conv2_block1_out[0][0]']
D)
conv2_block2_1_bn (BatchNo (None, 16, 16, 64) 256 ['conv2_block2_1_conv[0][0]']
rmalization)
conv2_block2_1_relu (Activ (None, 16, 16, 64) 0 ['conv2_block2_1_bn[0][0]']
ation)
conv2_block2_2_conv (Conv2 (None, 16, 16, 64) 36928 ['conv2_block2_1_relu[0][0]']
D)
conv2_block2_2_bn (BatchNo (None, 16, 16, 64) 256 ['conv2_block2_2_conv[0][0]']
rmalization)
conv2_block2_2_relu (Activ (None, 16, 16, 64) 0 ['conv2_block2_2_bn[0][0]']
ation)
conv2_block2_3_conv (Conv2 (None, 16, 16, 256) 16640 ['conv2_block2_2_relu[0][0]']
D)
conv2_block2_3_bn (BatchNo (None, 16, 16, 256) 1024 ['conv2_block2_3_conv[0][0]']
rmalization)
conv2_block2_add (Add) (None, 16, 16, 256) 0 ['conv2_block1_out[0][0]',
'conv2_block2_3_bn[0][0]']
conv2_block2_out (Activati (None, 16, 16, 256) 0 ['conv2_block2_add[0][0]']
on)
conv2_block3_1_conv (Conv2 (None, 16, 16, 64) 16448 ['conv2_block2_out[0][0]']
D)
conv2_block3_1_bn (BatchNo (None, 16, 16, 64) 256 ['conv2_block3_1_conv[0][0]']
rmalization)
conv2_block3_1_relu (Activ (None, 16, 16, 64) 0 ['conv2_block3_1_bn[0][0]']
ation)
conv2_block3_2_conv (Conv2 (None, 16, 16, 64) 36928 ['conv2_block3_1_relu[0][0]']
D)
conv2_block3_2_bn (BatchNo (None, 16, 16, 64) 256 ['conv2_block3_2_conv[0][0]']
rmalization)
conv2_block3_2_relu (Activ (None, 16, 16, 64) 0 ['conv2_block3_2_bn[0][0]']
ation)
conv2_block3_3_conv (Conv2 (None, 16, 16, 256) 16640 ['conv2_block3_2_relu[0][0]']
D)
conv2_block3_3_bn (BatchNo (None, 16, 16, 256) 1024 ['conv2_block3_3_conv[0][0]']
rmalization)
conv2_block3_add (Add) (None, 16, 16, 256) 0 ['conv2_block2_out[0][0]',
'conv2_block3_3_bn[0][0]']
conv2_block3_out (Activati (None, 16, 16, 256) 0 ['conv2_block3_add[0][0]']
on)
conv3_block1_1_conv (Conv2 (None, 8, 8, 128) 32896 ['conv2_block3_out[0][0]']
D)
conv3_block1_1_bn (BatchNo (None, 8, 8, 128) 512 ['conv3_block1_1_conv[0][0]']
rmalization)
conv3_block1_1_relu (Activ (None, 8, 8, 128) 0 ['conv3_block1_1_bn[0][0]']
ation)
conv3_block1_2_conv (Conv2 (None, 8, 8, 128) 147584 ['conv3_block1_1_relu[0][0]']
D)
conv3_block1_2_bn (BatchNo (None, 8, 8, 128) 512 ['conv3_block1_2_conv[0][0]']
rmalization)
conv3_block1_2_relu (Activ (None, 8, 8, 128) 0 ['conv3_block1_2_bn[0][0]']
ation)
conv3_block1_0_conv (Conv2 (None, 8, 8, 512) 131584 ['conv2_block3_out[0][0]']
D)
conv3_block1_3_conv (Conv2 (None, 8, 8, 512) 66048 ['conv3_block1_2_relu[0][0]']
D)
conv3_block1_0_bn (BatchNo (None, 8, 8, 512) 2048 ['conv3_block1_0_conv[0][0]']
rmalization)
conv3_block1_3_bn (BatchNo (None, 8, 8, 512) 2048 ['conv3_block1_3_conv[0][0]']
rmalization)
conv3_block1_add (Add) (None, 8, 8, 512) 0 ['conv3_block1_0_bn[0][0]',
'conv3_block1_3_bn[0][0]']
conv3_block1_out (Activati (None, 8, 8, 512) 0 ['conv3_block1_add[0][0]']
on)
conv3_block2_1_conv (Conv2 (None, 8, 8, 128) 65664 ['conv3_block1_out[0][0]']
D)
conv3_block2_1_bn (BatchNo (None, 8, 8, 128) 512 ['conv3_block2_1_conv[0][0]']
rmalization)
conv3_block2_1_relu (Activ (None, 8, 8, 128) 0 ['conv3_block2_1_bn[0][0]']
ation)
conv3_block2_2_conv (Conv2 (None, 8, 8, 128) 147584 ['conv3_block2_1_relu[0][0]']
D)
conv3_block2_2_bn (BatchNo (None, 8, 8, 128) 512 ['conv3_block2_2_conv[0][0]']
rmalization)
conv3_block2_2_relu (Activ (None, 8, 8, 128) 0 ['conv3_block2_2_bn[0][0]']
ation)
conv3_block2_3_conv (Conv2 (None, 8, 8, 512) 66048 ['conv3_block2_2_relu[0][0]']
D)
conv3_block2_3_bn (BatchNo (None, 8, 8, 512) 2048 ['conv3_block2_3_conv[0][0]']
rmalization)
conv3_block2_add (Add) (None, 8, 8, 512) 0 ['conv3_block1_out[0][0]',
'conv3_block2_3_bn[0][0]']
conv3_block2_out (Activati (None, 8, 8, 512) 0 ['conv3_block2_add[0][0]']
on)
conv3_block3_1_conv (Conv2 (None, 8, 8, 128) 65664 ['conv3_block2_out[0][0]']
D)
conv3_block3_1_bn (BatchNo (None, 8, 8, 128) 512 ['conv3_block3_1_conv[0][0]']
rmalization)
conv3_block3_1_relu (Activ (None, 8, 8, 128) 0 ['conv3_block3_1_bn[0][0]']
ation)
conv3_block3_2_conv (Conv2 (None, 8, 8, 128) 147584 ['conv3_block3_1_relu[0][0]']
D)
conv3_block3_2_bn (BatchNo (None, 8, 8, 128) 512 ['conv3_block3_2_conv[0][0]']
rmalization)
conv3_block3_2_relu (Activ (None, 8, 8, 128) 0 ['conv3_block3_2_bn[0][0]']
ation)
conv3_block3_3_conv (Conv2 (None, 8, 8, 512) 66048 ['conv3_block3_2_relu[0][0]']
D)
conv3_block3_3_bn (BatchNo (None, 8, 8, 512) 2048 ['conv3_block3_3_conv[0][0]']
rmalization)
conv3_block3_add (Add) (None, 8, 8, 512) 0 ['conv3_block2_out[0][0]',
'conv3_block3_3_bn[0][0]']
conv3_block3_out (Activati (None, 8, 8, 512) 0 ['conv3_block3_add[0][0]']
on)
conv3_block4_1_conv (Conv2 (None, 8, 8, 128) 65664 ['conv3_block3_out[0][0]']
D)
conv3_block4_1_bn (BatchNo (None, 8, 8, 128) 512 ['conv3_block4_1_conv[0][0]']
rmalization)
conv3_block4_1_relu (Activ (None, 8, 8, 128) 0 ['conv3_block4_1_bn[0][0]']
ation)
conv3_block4_2_conv (Conv2 (None, 8, 8, 128) 147584 ['conv3_block4_1_relu[0][0]']
D)
conv3_block4_2_bn (BatchNo (None, 8, 8, 128) 512 ['conv3_block4_2_conv[0][0]']
rmalization)
conv3_block4_2_relu (Activ (None, 8, 8, 128) 0 ['conv3_block4_2_bn[0][0]']
ation)
conv3_block4_3_conv (Conv2 (None, 8, 8, 512) 66048 ['conv3_block4_2_relu[0][0]']
D)
conv3_block4_3_bn (BatchNo (None, 8, 8, 512) 2048 ['conv3_block4_3_conv[0][0]']
rmalization)
conv3_block4_add (Add) (None, 8, 8, 512) 0 ['conv3_block3_out[0][0]',
'conv3_block4_3_bn[0][0]']
conv3_block4_out (Activati (None, 8, 8, 512) 0 ['conv3_block4_add[0][0]']
on)
conv4_block1_1_conv (Conv2 (None, 4, 4, 256) 131328 ['conv3_block4_out[0][0]']
D)
conv4_block1_1_bn (BatchNo (None, 4, 4, 256) 1024 ['conv4_block1_1_conv[0][0]']
rmalization)
conv4_block1_1_relu (Activ (None, 4, 4, 256) 0 ['conv4_block1_1_bn[0][0]']
ation)
conv4_block1_2_conv (Conv2 (None, 4, 4, 256) 590080 ['conv4_block1_1_relu[0][0]']
D)
conv4_block1_2_bn (BatchNo (None, 4, 4, 256) 1024 ['conv4_block1_2_conv[0][0]']
rmalization)
conv4_block1_2_relu (Activ (None, 4, 4, 256) 0 ['conv4_block1_2_bn[0][0]']
ation)
conv4_block1_0_conv (Conv2 (None, 4, 4, 1024) 525312 ['conv3_block4_out[0][0]']
D)
conv4_block1_3_conv (Conv2 (None, 4, 4, 1024) 263168 ['conv4_block1_2_relu[0][0]']
D)
conv4_block1_0_bn (BatchNo (None, 4, 4, 1024) 4096 ['conv4_block1_0_conv[0][0]']
rmalization)
conv4_block1_3_bn (BatchNo (None, 4, 4, 1024) 4096 ['conv4_block1_3_conv[0][0]']
rmalization)
conv4_block1_add (Add) (None, 4, 4, 1024) 0 ['conv4_block1_0_bn[0][0]',
'conv4_block1_3_bn[0][0]']
conv4_block1_out (Activati (None, 4, 4, 1024) 0 ['conv4_block1_add[0][0]']
on)
conv4_block2_1_conv (Conv2 (None, 4, 4, 256) 262400 ['conv4_block1_out[0][0]']
D)
conv4_block2_1_bn (BatchNo (None, 4, 4, 256) 1024 ['conv4_block2_1_conv[0][0]']
rmalization)
conv4_block2_1_relu (Activ (None, 4, 4, 256) 0 ['conv4_block2_1_bn[0][0]']
ation)
conv4_block2_2_conv (Conv2 (None, 4, 4, 256) 590080 ['conv4_block2_1_relu[0][0]']
D)
conv4_block2_2_bn (BatchNo (None, 4, 4, 256) 1024 ['conv4_block2_2_conv[0][0]']
rmalization)
conv4_block2_2_relu (Activ (None, 4, 4, 256) 0 ['conv4_block2_2_bn[0][0]']
ation)
conv4_block2_3_conv (Conv2 (None, 4, 4, 1024) 263168 ['conv4_block2_2_relu[0][0]']
D)
conv4_block2_3_bn (BatchNo (None, 4, 4, 1024) 4096 ['conv4_block2_3_conv[0][0]']
rmalization)
conv4_block2_add (Add) (None, 4, 4, 1024) 0 ['conv4_block1_out[0][0]',
'conv4_block2_3_bn[0][0]']
conv4_block2_out (Activati (None, 4, 4, 1024) 0 ['conv4_block2_add[0][0]']
on)
conv4_block3_1_conv (Conv2 (None, 4, 4, 256) 262400 ['conv4_block2_out[0][0]']
D)
conv4_block3_1_bn (BatchNo (None, 4, 4, 256) 1024 ['conv4_block3_1_conv[0][0]']
rmalization)
conv4_block3_1_relu (Activ (None, 4, 4, 256) 0 ['conv4_block3_1_bn[0][0]']
ation)
conv4_block3_2_conv (Conv2 (None, 4, 4, 256) 590080 ['conv4_block3_1_relu[0][0]']
D)
conv4_block3_2_bn (BatchNo (None, 4, 4, 256) 1024 ['conv4_block3_2_conv[0][0]']
rmalization)
conv4_block3_2_relu (Activ (None, 4, 4, 256) 0 ['conv4_block3_2_bn[0][0]']
ation)
conv4_block3_3_conv (Conv2 (None, 4, 4, 1024) 263168 ['conv4_block3_2_relu[0][0]']
D)
conv4_block3_3_bn (BatchNo (None, 4, 4, 1024) 4096 ['conv4_block3_3_conv[0][0]']
rmalization)
conv4_block3_add (Add) (None, 4, 4, 1024) 0 ['conv4_block2_out[0][0]',
'conv4_block3_3_bn[0][0]']
conv4_block3_out (Activati (None, 4, 4, 1024) 0 ['conv4_block3_add[0][0]']
on)
conv4_block4_1_conv (Conv2 (None, 4, 4, 256) 262400 ['conv4_block3_out[0][0]']
D)
conv4_block4_1_bn (BatchNo (None, 4, 4, 256) 1024 ['conv4_block4_1_conv[0][0]']
rmalization)
conv4_block4_1_relu (Activ (None, 4, 4, 256) 0 ['conv4_block4_1_bn[0][0]']
ation)
conv4_block4_2_conv (Conv2 (None, 4, 4, 256) 590080 ['conv4_block4_1_relu[0][0]']
D)
conv4_block4_2_bn (BatchNo (None, 4, 4, 256) 1024 ['conv4_block4_2_conv[0][0]']
rmalization)
conv4_block4_2_relu (Activ (None, 4, 4, 256) 0 ['conv4_block4_2_bn[0][0]']
ation)
conv4_block4_3_conv (Conv2 (None, 4, 4, 1024) 263168 ['conv4_block4_2_relu[0][0]']
D)
conv4_block4_3_bn (BatchNo (None, 4, 4, 1024) 4096 ['conv4_block4_3_conv[0][0]']
rmalization)
conv4_block4_add (Add) (None, 4, 4, 1024) 0 ['conv4_block3_out[0][0]',
'conv4_block4_3_bn[0][0]']
conv4_block4_out (Activati (None, 4, 4, 1024) 0 ['conv4_block4_add[0][0]']
on)
conv4_block5_1_conv (Conv2 (None, 4, 4, 256) 262400 ['conv4_block4_out[0][0]']
D)
conv4_block5_1_bn (BatchNo (None, 4, 4, 256) 1024 ['conv4_block5_1_conv[0][0]']
rmalization)
conv4_block5_1_relu (Activ (None, 4, 4, 256) 0 ['conv4_block5_1_bn[0][0]']
ation)
conv4_block5_2_conv (Conv2 (None, 4, 4, 256) 590080 ['conv4_block5_1_relu[0][0]']
D)
conv4_block5_2_bn (BatchNo (None, 4, 4, 256) 1024 ['conv4_block5_2_conv[0][0]']
rmalization)
conv4_block5_2_relu (Activ (None, 4, 4, 256) 0 ['conv4_block5_2_bn[0][0]']
ation)
conv4_block5_3_conv (Conv2 (None, 4, 4, 1024) 263168 ['conv4_block5_2_relu[0][0]']
D)
conv4_block5_3_bn (BatchNo (None, 4, 4, 1024) 4096 ['conv4_block5_3_conv[0][0]']
rmalization)
conv4_block5_add (Add) (None, 4, 4, 1024) 0 ['conv4_block4_out[0][0]',
'conv4_block5_3_bn[0][0]']
conv4_block5_out (Activati (None, 4, 4, 1024) 0 ['conv4_block5_add[0][0]']
on)
conv4_block6_1_conv (Conv2 (None, 4, 4, 256) 262400 ['conv4_block5_out[0][0]']
D)
conv4_block6_1_bn (BatchNo (None, 4, 4, 256) 1024 ['conv4_block6_1_conv[0][0]']
rmalization)
conv4_block6_1_relu (Activ (None, 4, 4, 256) 0 ['conv4_block6_1_bn[0][0]']
ation)
conv4_block6_2_conv (Conv2 (None, 4, 4, 256) 590080 ['conv4_block6_1_relu[0][0]']
D)
conv4_block6_2_bn (BatchNo (None, 4, 4, 256) 1024 ['conv4_block6_2_conv[0][0]']
rmalization)
conv4_block6_2_relu (Activ (None, 4, 4, 256) 0 ['conv4_block6_2_bn[0][0]']
ation)
conv4_block6_3_conv (Conv2 (None, 4, 4, 1024) 263168 ['conv4_block6_2_relu[0][0]']
D)
conv4_block6_3_bn (BatchNo (None, 4, 4, 1024) 4096 ['conv4_block6_3_conv[0][0]']
rmalization)
conv4_block6_add (Add) (None, 4, 4, 1024) 0 ['conv4_block5_out[0][0]',
'conv4_block6_3_bn[0][0]']
conv4_block6_out (Activati (None, 4, 4, 1024) 0 ['conv4_block6_add[0][0]']
on)
conv5_block1_1_conv (Conv2 (None, 2, 2, 512) 524800 ['conv4_block6_out[0][0]']
D)
conv5_block1_1_bn (BatchNo (None, 2, 2, 512) 2048 ['conv5_block1_1_conv[0][0]']
rmalization)
conv5_block1_1_relu (Activ (None, 2, 2, 512) 0 ['conv5_block1_1_bn[0][0]']
ation)
conv5_block1_2_conv (Conv2 (None, 2, 2, 512) 2359808 ['conv5_block1_1_relu[0][0]']
D)
conv5_block1_2_bn (BatchNo (None, 2, 2, 512) 2048 ['conv5_block1_2_conv[0][0]']
rmalization)
conv5_block1_2_relu (Activ (None, 2, 2, 512) 0 ['conv5_block1_2_bn[0][0]']
ation)
conv5_block1_0_conv (Conv2 (None, 2, 2, 2048) 2099200 ['conv4_block6_out[0][0]']
D)
conv5_block1_3_conv (Conv2 (None, 2, 2, 2048) 1050624 ['conv5_block1_2_relu[0][0]']
D)
conv5_block1_0_bn (BatchNo (None, 2, 2, 2048) 8192 ['conv5_block1_0_conv[0][0]']
rmalization)
conv5_block1_3_bn (BatchNo (None, 2, 2, 2048) 8192 ['conv5_block1_3_conv[0][0]']
rmalization)
conv5_block1_add (Add) (None, 2, 2, 2048) 0 ['conv5_block1_0_bn[0][0]',
'conv5_block1_3_bn[0][0]']
conv5_block1_out (Activati (None, 2, 2, 2048) 0 ['conv5_block1_add[0][0]']
on)
conv5_block2_1_conv (Conv2 (None, 2, 2, 512) 1049088 ['conv5_block1_out[0][0]']
D)
conv5_block2_1_bn (BatchNo (None, 2, 2, 512) 2048 ['conv5_block2_1_conv[0][0]']
rmalization)
conv5_block2_1_relu (Activ (None, 2, 2, 512) 0 ['conv5_block2_1_bn[0][0]']
ation)
conv5_block2_2_conv (Conv2 (None, 2, 2, 512) 2359808 ['conv5_block2_1_relu[0][0]']
D)
conv5_block2_2_bn (BatchNo (None, 2, 2, 512) 2048 ['conv5_block2_2_conv[0][0]']
rmalization)
conv5_block2_2_relu (Activ (None, 2, 2, 512) 0 ['conv5_block2_2_bn[0][0]']
ation)
conv5_block2_3_conv (Conv2 (None, 2, 2, 2048) 1050624 ['conv5_block2_2_relu[0][0]']
D)
conv5_block2_3_bn (BatchNo (None, 2, 2, 2048) 8192 ['conv5_block2_3_conv[0][0]']
rmalization)
conv5_block2_add (Add) (None, 2, 2, 2048) 0 ['conv5_block1_out[0][0]',
'conv5_block2_3_bn[0][0]']
conv5_block2_out (Activati (None, 2, 2, 2048) 0 ['conv5_block2_add[0][0]']
on)
conv5_block3_1_conv (Conv2 (None, 2, 2, 512) 1049088 ['conv5_block2_out[0][0]']
D)
conv5_block3_1_bn (BatchNo (None, 2, 2, 512) 2048 ['conv5_block3_1_conv[0][0]']
rmalization)
conv5_block3_1_relu (Activ (None, 2, 2, 512) 0 ['conv5_block3_1_bn[0][0]']
ation)
conv5_block3_2_conv (Conv2 (None, 2, 2, 512) 2359808 ['conv5_block3_1_relu[0][0]']
D)
conv5_block3_2_bn (BatchNo (None, 2, 2, 512) 2048 ['conv5_block3_2_conv[0][0]']
rmalization)
conv5_block3_2_relu (Activ (None, 2, 2, 512) 0 ['conv5_block3_2_bn[0][0]']
ation)
conv5_block3_3_conv (Conv2 (None, 2, 2, 2048) 1050624 ['conv5_block3_2_relu[0][0]']
D)
conv5_block3_3_bn (BatchNo (None, 2, 2, 2048) 8192 ['conv5_block3_3_conv[0][0]']
rmalization)
conv5_block3_add (Add) (None, 2, 2, 2048) 0 ['conv5_block2_out[0][0]',
'conv5_block3_3_bn[0][0]']
conv5_block3_out (Activati (None, 2, 2, 2048) 0 ['conv5_block3_add[0][0]']
on)
max_pool (GlobalMaxPooling (None, 2048) 0 ['conv5_block3_out[0][0]']
2D)
batch_normalization_4 (Bat (None, 2048) 8192 ['max_pool[0][0]']
chNormalization)
dense_18 (Dense) (None, 512) 1049088 ['batch_normalization_4[0][0]'
]
dropout_8 (Dropout) (None, 512) 0 ['dense_18[0][0]']
dense_19 (Dense) (None, 102) 52326 ['dropout_8[0][0]']
==================================================================================================
Total params: 24697318 (94.21 MB)
Trainable params: 24538406 (93.61 MB)
Non-trainable params: 158912 (620.75 KB)
__________________________________________________________________________________________________
with tf.device('/device:GPU:0'):
resnet_history = model.fit(trainimagedata,
validation_data = valimagedata,
epochs= 5,
batch_size=512,
callbacks=[early_stop])
Epoch 1/5 8/8 [==============================] - 33s 4s/step - loss: 19.0899 - accuracy: 0.0118 - val_loss: 1170.8286 - val_accuracy: 0.0105 Epoch 2/5 8/8 [==============================] - 29s 4s/step - loss: 15.4293 - accuracy: 0.0105 - val_loss: 67866.1406 - val_accuracy: 0.0053 Epoch 3/5 8/8 [==============================] - 28s 4s/step - loss: 13.4399 - accuracy: 0.0100 - val_loss: 215974.2812 - val_accuracy: 0.0068 Epoch 4/5 8/8 [==============================] - 29s 4s/step - loss: 11.5912 - accuracy: 0.0130 - val_loss: 100673.8594 - val_accuracy: 0.0226
plt.plot(resnet_history.history['loss'])
plt.plot(resnet_history.history['val_loss'])
plt.title('ResNet50 - Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.show()
plt.plot(resnet_history.history['accuracy'])
plt.plot(resnet_history.history['val_accuracy'])
plt.title('ResNet50 - Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.show()
model.save("/Users/lmschwenke/Downloads/bugs/resnet_model.h5")
/Users/lmschwenke/miniconda3/lib/python3.10/site-packages/keras/src/engine/training.py:3000: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.
saving_api.save_model(
CNN Accuracy: The accuracy over epochs for the Custom CNN model starts to increase sharply after epoch 2. Both train and test sets follow similar patterns with similar results. Overall the model accuracy is low in the single-digits. Continuous iteration on this model would help capture more of the underlying patterns.
ResNet50 Accuracy: The accuracy over epochs for the ResNet50 model has a dip in the test accuracy that then increases at epoch 2. The results were farily similar to the Custom CNN model. The train accuracy remained pretty static whereas the test accuracy increased.
Performance Comparison: Both models had similar performance from an accuracy standpoint (around 2%). This dataset has 102 classes to classify and we subsetted the data to a much smaller number of images to balances the classes and allow the models to train in a reasonable time on local hardware. Future iterations that include the full dataset, larger computer resources on a TPU/GPU, inceased training time with epochs, and more refined model architectures would benefit this dataset classification.
Test Example (Web Application - below): The test example within the web application shows the Custom CNN model classifying the bug picture as 65% Dacus Dorsalis (Hendel) and 33% Nipaecoccus Vastalor. The ResNet50 model on the other hand just predicted 100% to the white margined moth.
The web application was created in VScode, the files are attached separately. Please see the screenshots below showing an example process of an image being submitted and the prediction from the trained model.
The application allows the user to select the Custom CNN or ResNet model to make the prediction on their uploaded image. The page dynamically updates and runs the appropriate model.
Build two RNNs (an LSTM and GRU) models each to learn the Sine wave and the arbitrary function from Assignment 2 (20)
# Sine
import numpy as np
X = np.linspace(-np.pi, np.pi, 10000)
X = X.reshape(-1, 1)
X = X.reshape(-1, 1, 1)
y = np.sin(X)
y = y.reshape(-1, 1)
tsteps = 1
batch_size = 25
epochs = 100
model = Sequential()
model.add(LSTM(50, batch_input_shape=(batch_size, tsteps, 1), return_sequences=True, stateful=True))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
print("Model Parameters :", model.count_params())
print("Output Shape :", model.output_shape)
model.summary()
Model Parameters : 10451
Output Shape : (25, 1, 1)
Model: "sequential_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
lstm (LSTM) (25, 1, 50) 10400
dense_5 (Dense) (25, 1, 1) 51
=================================================================
Total params: 10451 (40.82 KB)
Trainable params: 10451 (40.82 KB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________
#Train an LSTM model
print('Training')
for i in range(epochs):
print('Epoch', i, '/', epochs)
model.fit(X,
y,
batch_size=batch_size,
verbose=1,
epochs=1,
shuffle=False)
model.reset_states()
Training Epoch 0 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0056 Epoch 1 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0099 Epoch 2 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0335 Epoch 3 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0193 Epoch 4 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0135 Epoch 5 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0095 Epoch 6 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0104 Epoch 7 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0118 Epoch 8 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0161 Epoch 9 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0187 Epoch 10 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0206 Epoch 11 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0235 Epoch 12 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0259 Epoch 13 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0165 Epoch 14 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0220 Epoch 15 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0246 Epoch 16 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0182 Epoch 17 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0239 Epoch 18 / 100 400/400 [==============================] - 2s 5ms/step - loss: 0.0215 Epoch 19 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0273 Epoch 20 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0222 Epoch 21 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0253 Epoch 22 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0170 Epoch 23 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0199 Epoch 24 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0136 Epoch 25 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0125 Epoch 26 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0033 Epoch 27 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0049 Epoch 28 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0019 Epoch 29 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0038 Epoch 30 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0056 Epoch 31 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0072 Epoch 32 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0298 Epoch 33 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0117 Epoch 34 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0039 Epoch 35 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0054 Epoch 36 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0030 Epoch 37 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0099 Epoch 38 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0012 Epoch 39 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0036 Epoch 40 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0025 Epoch 41 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0010 Epoch 42 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0028 Epoch 43 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.0865e-04 Epoch 44 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.9277e-04 Epoch 45 / 100 400/400 [==============================] - 2s 4ms/step - loss: 6.1654e-04 Epoch 46 / 100 400/400 [==============================] - 2s 4ms/step - loss: 6.5924e-04 Epoch 47 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.6743e-04 Epoch 48 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.4149e-04 Epoch 49 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.1666e-04 Epoch 50 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.3196e-04 Epoch 51 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.3208e-04 Epoch 52 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.7546e-04 Epoch 53 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.9139e-04 Epoch 54 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.0862e-04 Epoch 55 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.2490e-04 Epoch 56 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.8662e-04 Epoch 57 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.2256e-04 Epoch 58 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.7822e-04 Epoch 59 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.7526e-04 Epoch 60 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.8805e-04 Epoch 61 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.7578e-04 Epoch 62 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.5610e-04 Epoch 63 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.6966e-04 Epoch 64 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.8376e-04 Epoch 65 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0012 Epoch 66 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.3531e-04 Epoch 67 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.4019e-04 Epoch 68 / 100 400/400 [==============================] - 2s 4ms/step - loss: 6.1081e-04 Epoch 69 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0026 Epoch 70 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.7666e-04 Epoch 71 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0015 Epoch 72 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0036 Epoch 73 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0018 Epoch 74 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0013 Epoch 75 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.5525e-04 Epoch 76 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0012 Epoch 77 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0029 Epoch 78 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0015 Epoch 79 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0030 Epoch 80 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0082 Epoch 81 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0019 Epoch 82 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0033 Epoch 83 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0033 Epoch 84 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.9521e-04 Epoch 85 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.4930e-04 Epoch 86 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0014 Epoch 87 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.7115e-04 Epoch 88 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.7361e-04 Epoch 89 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.5529e-04 Epoch 90 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.7995e-04 Epoch 91 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.7455e-04 Epoch 92 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.8635e-04 Epoch 93 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0011 Epoch 94 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.4261e-04 Epoch 95 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0012 Epoch 96 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0026 Epoch 97 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0015 Epoch 98 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0016 Epoch 99 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0018
print('Predicting')
predicted_output = model.predict(X, batch_size=batch_size)
predicted_output = np.squeeze(predicted_output, axis=(2,))
predicted_output.shape
y.shape #expected output
Predicting 400/400 [==============================] - 1s 2ms/step
(10000, 1)
plt.figure(figsize=(20,5))
plt.plot(y, "red", label='Train')
plt.plot(predicted_output, "green", label='Test')
plt.title('Sine Wave - Expected and Predicted Outputs (using LSTM)')
plt.legend()
plt.show()
model = Sequential()
model.add(GRU(50, batch_input_shape=(batch_size, tsteps, 1), return_sequences=True, stateful=True))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
model.summary()
Model: "sequential_2"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
gru (GRU) (25, 1, 50) 7950
dense_6 (Dense) (25, 1, 1) 51
=================================================================
Total params: 8001 (31.25 KB)
Trainable params: 8001 (31.25 KB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________
#Train an GRU model
print('Training')
for i in range(epochs):
print('Epoch', i, '/', epochs)
model.fit(X,
y,
batch_size=batch_size,
verbose=1,
epochs=1,
shuffle=False)
model.reset_states()
Training Epoch 0 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0023 Epoch 1 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0010 Epoch 2 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0176 Epoch 3 / 100 400/400 [==============================] - 1s 4ms/step - loss: 0.0219 Epoch 4 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0204 Epoch 5 / 100 400/400 [==============================] - 1s 4ms/step - loss: 0.0176 Epoch 6 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0195 Epoch 7 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0120 Epoch 8 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0134 Epoch 9 / 100 400/400 [==============================] - 1s 4ms/step - loss: 0.0082 Epoch 10 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0248 Epoch 11 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0091 Epoch 12 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0048 Epoch 13 / 100 400/400 [==============================] - 1s 4ms/step - loss: 0.0236 Epoch 14 / 100 400/400 [==============================] - 1s 4ms/step - loss: 0.0064 Epoch 15 / 100 400/400 [==============================] - 1s 4ms/step - loss: 0.0024 Epoch 16 / 100 400/400 [==============================] - 1s 4ms/step - loss: 0.0038 Epoch 17 / 100 400/400 [==============================] - 1s 4ms/step - loss: 0.0092 Epoch 18 / 100 400/400 [==============================] - 1s 4ms/step - loss: 0.0072 Epoch 19 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0136 Epoch 20 / 100 400/400 [==============================] - 1s 4ms/step - loss: 0.0069 Epoch 21 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0014 Epoch 22 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0027 Epoch 23 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0013 Epoch 24 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0013 Epoch 25 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0019 Epoch 26 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0027 Epoch 27 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0037 Epoch 28 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0018 Epoch 29 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0048 Epoch 30 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.8622e-04 Epoch 31 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0016 Epoch 32 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0017 Epoch 33 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.1324e-04 Epoch 34 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.1494e-04 Epoch 35 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0025 Epoch 36 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0046 Epoch 37 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0032 Epoch 38 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0021 Epoch 39 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0053 Epoch 40 / 100 400/400 [==============================] - 1s 4ms/step - loss: 0.0084 Epoch 41 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0061 Epoch 42 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0075 Epoch 43 / 100 400/400 [==============================] - 1s 4ms/step - loss: 0.0039 Epoch 44 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.8876e-04 Epoch 45 / 100 400/400 [==============================] - 2s 4ms/step - loss: 6.2033e-04 Epoch 46 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0012 Epoch 47 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0011 Epoch 48 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.8414e-04 Epoch 49 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.9681e-04 Epoch 50 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.8676e-05 Epoch 51 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.2682e-04 Epoch 52 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.7899e-04 Epoch 53 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.7397e-04 Epoch 54 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.6097e-04 Epoch 55 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.7269e-04 Epoch 56 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.2354e-04 Epoch 57 / 100 400/400 [==============================] - 1s 4ms/step - loss: 2.6462e-04 Epoch 58 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.0712e-05 Epoch 59 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.7932e-04 Epoch 60 / 100 400/400 [==============================] - 1s 4ms/step - loss: 2.0097e-05 Epoch 61 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.1077e-05 Epoch 62 / 100 400/400 [==============================] - 1s 4ms/step - loss: 7.3675e-05 Epoch 63 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.1493e-04 Epoch 64 / 100 400/400 [==============================] - 1s 4ms/step - loss: 2.3827e-05 Epoch 65 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.9735e-04 Epoch 66 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.9705e-04 Epoch 67 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0015 Epoch 68 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.0548e-04 Epoch 69 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.6877e-04 Epoch 70 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.9102e-05 Epoch 71 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.8940e-05 Epoch 72 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.3757e-05 Epoch 73 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.3591e-05 Epoch 74 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.6441e-05 Epoch 75 / 100 400/400 [==============================] - 1s 4ms/step - loss: 5.5253e-04 Epoch 76 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.4781e-04 Epoch 77 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.4475e-05 Epoch 78 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.3518e-05 Epoch 79 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.4776e-04 Epoch 80 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.5325e-05 Epoch 81 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.0006e-05 Epoch 82 / 100 400/400 [==============================] - 1s 4ms/step - loss: 1.0748e-04 Epoch 83 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.3957e-05 Epoch 84 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.4708e-05 Epoch 85 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.7491e-04 Epoch 86 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.0652e-05 Epoch 87 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.4202e-05 Epoch 88 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.4663e-05 Epoch 89 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.7405e-05 Epoch 90 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.0192e-05 Epoch 91 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.4759e-05 Epoch 92 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.4924e-06 Epoch 93 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.9340e-05 Epoch 94 / 100 400/400 [==============================] - 1s 4ms/step - loss: 1.1762e-04 Epoch 95 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.2023e-04 Epoch 96 / 100 400/400 [==============================] - 1s 4ms/step - loss: 5.1391e-04 Epoch 97 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.2697e-04 Epoch 98 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.6314e-04 Epoch 99 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.5712e-04
print('Predicting')
predicted_output = model.predict(X, batch_size=batch_size)
predicted_output = np.squeeze(predicted_output, axis=(2,))
predicted_output.shape
y.shape
Predicting 400/400 [==============================] - 1s 1ms/step
(10000, 1)
plt.figure(figsize=(20,5))
plt.plot(y, "red", label='Train')
plt.plot(predicted_output, "green", label='Test')
plt.title('Sine Wave - Expected and Predicted Outputs (using GRU)')
plt.legend()
plt.show()
# Sigmoid
X = np.linspace(-np.pi, np.pi, 10000)
X = X.reshape(-1, 1)
X = X.reshape(-1, 1, 1)
y = 1 / (1 + np.exp(-X))
y = y.reshape(-1, 1)
model = Sequential()
model.add(LSTM(50, batch_input_shape=(batch_size, tsteps, 1), return_sequences=True, stateful=True))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
print("Model Parameters :", model.count_params())
print("Output Shape :", model.output_shape)
model.summary()
Model Parameters : 10451
Output Shape : (25, 1, 1)
Model: "sequential_3"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
lstm_1 (LSTM) (25, 1, 50) 10400
dense_7 (Dense) (25, 1, 1) 51
=================================================================
Total params: 10451 (40.82 KB)
Trainable params: 10451 (40.82 KB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________
#Train an LSTM model
print('Training')
for i in range(epochs):
print('Epoch', i, '/', epochs)
model.fit(X,
y,
batch_size=batch_size,
verbose=1,
epochs=1,
shuffle=False)
model.reset_states()
Training Epoch 0 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0350 Epoch 1 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0038 Epoch 2 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0036 Epoch 3 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.4753e-04 Epoch 4 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.1990e-04 Epoch 5 / 100 400/400 [==============================] - 2s 5ms/step - loss: 8.5542e-04 Epoch 6 / 100 400/400 [==============================] - 2s 4ms/step - loss: 6.0169e-04 Epoch 7 / 100 400/400 [==============================] - 2s 4ms/step - loss: 6.0030e-04 Epoch 8 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.4323e-04 Epoch 9 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.5605e-04 Epoch 10 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.5139e-04 Epoch 11 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.7270e-04 Epoch 12 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.4719e-04 Epoch 13 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.3018e-04 Epoch 14 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.8048e-04 Epoch 15 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.8506e-04 Epoch 16 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.1898e-05 Epoch 17 / 100 400/400 [==============================] - 2s 4ms/step - loss: 6.3949e-04 Epoch 18 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.5376e-04 Epoch 19 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.6681e-04 Epoch 20 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.0367e-04 Epoch 21 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.0560e-04 Epoch 22 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.1659e-04 Epoch 23 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.1049e-05 Epoch 24 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.5879e-05 Epoch 25 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.1740e-04 Epoch 26 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.0644e-04 Epoch 27 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.4098e-04 Epoch 28 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.1042e-04 Epoch 29 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.8991e-05 Epoch 30 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.0649e-04 Epoch 31 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.9050e-05 Epoch 32 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.0582e-04 Epoch 33 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.7832e-05 Epoch 34 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.0189e-05 Epoch 35 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.1778e-05 Epoch 36 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.5333e-04 Epoch 37 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.3780e-05 Epoch 38 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.1081e-04 Epoch 39 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.9766e-05 Epoch 40 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.1929e-05 Epoch 41 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.4002e-05 Epoch 42 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.5585e-05 Epoch 43 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.0669e-04 Epoch 44 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.3585e-05 Epoch 45 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.6148e-05 Epoch 46 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.0040e-05 Epoch 47 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.3255e-06 Epoch 48 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.5794e-05 Epoch 49 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.3439e-05 Epoch 50 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.1207e-05 Epoch 51 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.9579e-05 Epoch 52 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.2839e-05 Epoch 53 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.8181e-05 Epoch 54 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.0499e-05 Epoch 55 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.9259e-05 Epoch 56 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.4501e-05 Epoch 57 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.6913e-05 Epoch 58 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.6566e-05 Epoch 59 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.8089e-05 Epoch 60 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.8481e-06 Epoch 61 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.7060e-05 Epoch 62 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.1533e-05 Epoch 63 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.3934e-05 Epoch 64 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.3251e-05 Epoch 65 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.7248e-06 Epoch 66 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.5719e-06 Epoch 67 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.3474e-05 Epoch 68 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.7427e-07 Epoch 69 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.2854e-05 Epoch 70 / 100 400/400 [==============================] - 2s 4ms/step - loss: 6.8500e-06 Epoch 71 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.1774e-06 Epoch 72 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.0756e-06 Epoch 73 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.6437e-05 Epoch 74 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.7130e-05 Epoch 75 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.0538e-05 Epoch 76 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.5515e-06 Epoch 77 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.4143e-05 Epoch 78 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.2962e-05 Epoch 79 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.1067e-05 Epoch 80 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.8424e-06 Epoch 81 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.8953e-06 Epoch 82 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.6055e-05 Epoch 83 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.6637e-06 Epoch 84 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.2039e-05 Epoch 85 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.8272e-06 Epoch 86 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.3654e-06 Epoch 87 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.0096e-05 Epoch 88 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.1847e-05 Epoch 89 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.6449e-06 Epoch 90 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.4905e-05 Epoch 91 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.0430e-06 Epoch 92 / 100 400/400 [==============================] - 1s 4ms/step - loss: 3.6700e-06 Epoch 93 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.9275e-06 Epoch 94 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.2837e-06 Epoch 95 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.0237e-05 Epoch 96 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.2620e-06 Epoch 97 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.8207e-05 Epoch 98 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.1561e-06 Epoch 99 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.3567e-06
print('Predicting')
predicted_output = model.predict(X, batch_size=batch_size)
predicted_output = np.squeeze(predicted_output, axis=(2,))
predicted_output.shape
y.shape #expected output
Predicting 400/400 [==============================] - 1s 2ms/step
(10000, 1)
plt.figure(figsize=(20,5))
plt.plot(y, "red", label='Train')
plt.plot(predicted_output, "green", label='Test')
plt.title('Sigmoid - Expected and Predicted Outputs (using LSTM)')
plt.legend()
plt.show()
model = Sequential()
model.add(GRU(50, batch_input_shape=(batch_size, tsteps, 1), return_sequences=True, stateful=True))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
print("Model Parameters :", model.count_params())
print("Output Shape :", model.output_shape)
model.summary()
Model Parameters : 8001
Output Shape : (25, 1, 1)
Model: "sequential_4"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
gru_1 (GRU) (25, 1, 50) 7950
dense_8 (Dense) (25, 1, 1) 51
=================================================================
Total params: 8001 (31.25 KB)
Trainable params: 8001 (31.25 KB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________
#Train a GRU model
print('Training')
for i in range(epochs):
print('Epoch', i, '/', epochs)
model.fit(X,
y,
batch_size=batch_size,
verbose=1,
epochs=1,
shuffle=False)
model.reset_states()
Training Epoch 0 / 100 400/400 [==============================] - 2s 4ms/step - loss: 0.0276 Epoch 1 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.9183e-04 Epoch 2 / 100 400/400 [==============================] - 1s 4ms/step - loss: 3.6255e-04 Epoch 3 / 100 400/400 [==============================] - 1s 4ms/step - loss: 1.9363e-04 Epoch 4 / 100 400/400 [==============================] - 1s 4ms/step - loss: 9.4758e-05 Epoch 5 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.8416e-05 Epoch 6 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.5445e-04 Epoch 7 / 100 400/400 [==============================] - 1s 4ms/step - loss: 1.1527e-04 Epoch 8 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.5127e-04 Epoch 9 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.4645e-04 Epoch 10 / 100 400/400 [==============================] - 1s 4ms/step - loss: 9.1154e-05 Epoch 11 / 100 400/400 [==============================] - 1s 4ms/step - loss: 4.0214e-05 Epoch 12 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.4728e-04 Epoch 13 / 100 400/400 [==============================] - 1s 4ms/step - loss: 4.1717e-04 Epoch 14 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.4679e-04 Epoch 15 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.4477e-04 Epoch 16 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.0936e-05 Epoch 17 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.4122e-05 Epoch 18 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.6008e-04 Epoch 19 / 100 400/400 [==============================] - 1s 4ms/step - loss: 5.0254e-05 Epoch 20 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.5419e-04 Epoch 21 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.7165e-05 Epoch 22 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.6610e-04 Epoch 23 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.3917e-04 Epoch 24 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.4333e-05 Epoch 25 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.5789e-04 Epoch 26 / 100 400/400 [==============================] - 1s 4ms/step - loss: 2.0351e-04 Epoch 27 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.5202e-04 Epoch 28 / 100 400/400 [==============================] - 1s 4ms/step - loss: 6.9288e-05 Epoch 29 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.3576e-05 Epoch 30 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.9201e-05 Epoch 31 / 100 400/400 [==============================] - 1s 4ms/step - loss: 1.0891e-04 Epoch 32 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.0260e-05 Epoch 33 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.6800e-05 Epoch 34 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.1139e-04 Epoch 35 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.7709e-05 Epoch 36 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.0875e-05 Epoch 37 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.9694e-05 Epoch 38 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.9986e-05 Epoch 39 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.4187e-05 Epoch 40 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.6055e-04 Epoch 41 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.6095e-04 Epoch 42 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.1900e-05 Epoch 43 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.1776e-04 Epoch 44 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.8634e-04 Epoch 45 / 100 400/400 [==============================] - 1s 4ms/step - loss: 2.9904e-04 Epoch 46 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.7391e-04 Epoch 47 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.3585e-04 Epoch 48 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.7183e-04 Epoch 49 / 100 400/400 [==============================] - 1s 4ms/step - loss: 4.0148e-04 Epoch 50 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.3933e-04 Epoch 51 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.4968e-04 Epoch 52 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.1816e-04 Epoch 53 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.2267e-04 Epoch 54 / 100 400/400 [==============================] - 2s 4ms/step - loss: 6.2151e-04 Epoch 55 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.3656e-05 Epoch 56 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.9482e-05 Epoch 57 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.5243e-05 Epoch 58 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.0776e-05 Epoch 59 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.1498e-04 Epoch 60 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.4873e-04 Epoch 61 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.5594e-05 Epoch 62 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.8123e-04 Epoch 63 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.9237e-06 Epoch 64 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.0878e-06 Epoch 65 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.4402e-05 Epoch 66 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.1083e-05 Epoch 67 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.0830e-05 Epoch 68 / 100 400/400 [==============================] - 2s 4ms/step - loss: 6.3598e-05 Epoch 69 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.1803e-05 Epoch 70 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.8105e-04 Epoch 71 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.9541e-05 Epoch 72 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.2236e-05 Epoch 73 / 100 400/400 [==============================] - 2s 4ms/step - loss: 3.9898e-05 Epoch 74 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.2402e-04 Epoch 75 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.5073e-05 Epoch 76 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.9836e-05 Epoch 77 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.6709e-04 Epoch 78 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.2663e-04 Epoch 79 / 100 400/400 [==============================] - 2s 4ms/step - loss: 8.6777e-05 Epoch 80 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.0123e-05 Epoch 81 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.2075e-05 Epoch 82 / 100 400/400 [==============================] - 2s 4ms/step - loss: 9.9261e-06 Epoch 83 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.1150e-05 Epoch 84 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.6662e-05 Epoch 85 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.8779e-06 Epoch 86 / 100 400/400 [==============================] - 2s 4ms/step - loss: 7.9059e-06 Epoch 87 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.3102e-05 Epoch 88 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.8141e-05 Epoch 89 / 100 400/400 [==============================] - 2s 4ms/step - loss: 2.7707e-05 Epoch 90 / 100 400/400 [==============================] - 2s 4ms/step - loss: 6.8681e-05 Epoch 91 / 100 400/400 [==============================] - 2s 4ms/step - loss: 1.7901e-05 Epoch 92 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.2943e-05 Epoch 93 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.0642e-05 Epoch 94 / 100 400/400 [==============================] - 1s 4ms/step - loss: 1.2274e-04 Epoch 95 / 100 400/400 [==============================] - 1s 4ms/step - loss: 2.7727e-05 Epoch 96 / 100 400/400 [==============================] - 2s 4ms/step - loss: 5.4640e-05 Epoch 97 / 100 400/400 [==============================] - 1s 4ms/step - loss: 1.9389e-05 Epoch 98 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.7264e-05 Epoch 99 / 100 400/400 [==============================] - 2s 4ms/step - loss: 4.1462e-05
print('Predicting')
predicted_output = model.predict(X, batch_size=batch_size)
predicted_output = np.squeeze(predicted_output, axis=(2,))
predicted_output.shape
y.shape #expected output
Predicting 400/400 [==============================] - 1s 1ms/step
(10000, 1)
plt.figure(figsize=(20,5))
plt.plot(y, "red", label='Train')
plt.plot(predicted_output, "green", label='Test')
plt.title('Sigmoid - Expected and Predicted Outputs (using GRU)')
plt.legend()
plt.show()
LSTM Model: This model consists of a single LSTM layer and a dense layer.
GRU Model: This model consists of a single GRU layer and a dense layer.
LSTM Parameter Count Formula = 𝑚:𝑠𝑖𝑧𝑒𝑜𝑓𝑖𝑛𝑝𝑢𝑡, 𝑛:𝑠𝑖𝑧𝑒𝑜𝑓𝑜𝑢𝑡𝑝𝑢𝑡 • 𝑝𝑎𝑟𝑎𝑚𝑒𝑡𝑒𝑟𝑠 = 4(𝑛𝑚 + 𝑛^2 + 𝑛)
LSTM Model Paramaters: 10,451
Layer 1 (LSTM)
Layer 2 (Dense)
Total number of parameters in the GRU RNN equals 3(𝑛^2+𝑛𝑚+2𝑛) where 𝑚 is the input dimension and 𝑛 is the output dimension. GRU Parameter Count Formula = 3(𝑛^2+𝑛*𝑚+𝑛) m: intput, n: output
GRU Model Parameters: 8,001
Layer 1 (GRU) = 3 * (50^2 + 50(1) + 2(50)) = 7,950
Layer 2 (Dense) = (25, 1, 1) = 50 (input size) 1 (output size) + 1 (bias) 1 (output size) = 50 + 1 = 51
Total parameters = 7,950 + 51 = 8,001
Compared to my MLP model performance in assignment to, the LSTM and GRU models do not perform as well. This is evident in the MLP plots below compared to the LSTM/GRU plots above. The new models are not fitting the Sine nor the Sigmoid functions nearly as well. This may be solved be increasing the model complexities and epochs to learn more patterns, but overall the MLP has strong performance on these functions with minimal complexity.
